{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda\\lib\\site-packages\\sklearn\\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import xgboost as xgb\n",
    "from sklearn.cross_validation import KFold\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression,RidgeClassifier,Ridge\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "seed = 1024\n",
    "NFOLDS = 5\n",
    "np.random.seed(seed)\n",
    "\n",
    "path = '../data/'\n",
    "\n",
    "train_x = pd.read_pickle(path + 'train_X.pkl')\n",
    "valid_x = pd.read_pickle(path + 'valid_X.pkl')\n",
    "dev_x = pd.read_pickle(path + 'dev_X.pkl')\n",
    "\n",
    "y_train = pd.read_pickle(path+'train.pkl')['label'].values\n",
    "y_valid = pd.read_pickle(path+'valid.pkl')['label'].values\n",
    "y_dev = pd.read_pickle(path+'dev.pkl')['label'].values\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ntrain = train_x.shape[0]\n",
    "ntest = valid_x.shape[0]\n",
    "\n",
    "\n",
    "x_train = np.array(train_x)\n",
    "x_test = np.array(valid_x)\n",
    "\n",
    "\n",
    "kf = KFold(ntrain, n_folds=NFOLDS, shuffle=True, random_state=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class SklearnWrapper(object):\n",
    "    def __init__(self, clf, seed=0, params=None):\n",
    "        params['random_state'] = seed\n",
    "        self.clf = clf(**params)\n",
    "\n",
    "    def train(self, x_train, y_train):\n",
    "        self.clf.fit(x_train, y_train)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return self.clf.predict(x)\n",
    "\n",
    "\n",
    "class XgbWrapper(object):\n",
    "    def __init__(self, seed=0, params=None):\n",
    "        self.param = params\n",
    "        self.param['seed'] = seed\n",
    "        self.nrounds = params.pop('nrounds', 500)\n",
    "\n",
    "    def train(self, x_train, y_train):\n",
    "        dtrain = xgb.DMatrix(x_train, label=y_train)\n",
    "        self.gbdt = xgb.train(self.param, dtrain, self.nrounds)\n",
    "\n",
    "    def predict(self, x):\n",
    "        return self.gbdt.predict(xgb.DMatrix(x))\n",
    "\n",
    "\n",
    "def get_oof(clf):\n",
    "    oof_train = np.zeros((ntrain,))\n",
    "    oof_test = np.zeros((ntest,))\n",
    "    oof_test_skf = np.empty((NFOLDS, ntest))\n",
    "\n",
    "    for i, (train_index, test_index) in enumerate(kf):\n",
    "        x_tr = x_train[train_index]\n",
    "        y_tr = y_train[train_index]\n",
    "        x_te = x_train[test_index]\n",
    "\n",
    "        clf.train(x_tr, y_tr)\n",
    "\n",
    "        oof_train[test_index] = clf.predict(x_te)#use for train\n",
    "        oof_test_skf[i, :] = clf.predict(x_test)# use for test\n",
    "\n",
    "    oof_test[:] = oof_test_skf.mean(axis=0)\n",
    "    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "et_params = {\n",
    "    'n_jobs': 16,\n",
    "    'n_estimators': 250,\n",
    "    'max_features': 0.5,\n",
    "    'max_depth': 10,\n",
    "    'min_samples_split':20,\n",
    "    'min_samples_leaf':10,\n",
    "\n",
    "}\n",
    "\n",
    "rf_params = {\n",
    "    'n_jobs': 16,\n",
    "    'n_estimators': 250,\n",
    "    'max_features': 0.5,\n",
    "    'max_depth': 8,\n",
    "    'min_samples_leaf': 2,\n",
    "}\n",
    "\n",
    "xgb_params = {\n",
    "    'max_depth':8,\n",
    "    'nthread':18,\n",
    "    'eta':0.03,\n",
    "    'eval_metric':['error','logloss'],\n",
    "    #'eval_metric':['logloss','error'],\n",
    "    'objective':'binary:logistic',\n",
    "    'subsample':0.7,\n",
    "    'colsample_bytree':0.5,\n",
    "    'silent':1,\n",
    "    'seed':1123,\n",
    "    'min_child_weight':10,\n",
    "    'nrounds': 400\n",
    "}\n",
    "\n",
    "xgb_params_2 = {\n",
    "    'max_depth':7,\n",
    "    'nthread':18,\n",
    "    'eta':0.03,\n",
    "    'eval_metric':['error','logloss'],\n",
    "    #'eval_metric':['logloss','error'],\n",
    "    'objective':'binary:logistic',\n",
    "    'subsample':0.5,\n",
    "    'colsample_bytree':0.5,\n",
    "    'silent':1,\n",
    "    'seed':1123,\n",
    "    'min_child_weight':10,\n",
    "    'nrounds': 400\n",
    "}\n",
    "\n",
    "lr_params = {\n",
    "    'C':1.0,\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xg = XgbWrapper(seed=seed, params=xgb_params)\n",
    "xg2 = XgbWrapper(seed=seed, params=xgb_params_2)\n",
    "et = SklearnWrapper(clf=ExtraTreesClassifier, seed=seed, params=et_params)\n",
    "rf = SklearnWrapper(clf=RandomForestClassifier, seed=seed, params=rf_params)\n",
    "Lr = SklearnWrapper(clf=LogisticRegression,seed=seed,params=lr_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "xg_oof_train, xg_oof_test = get_oof(xg)\n",
    "xg_oof_train_2, xg_oof_test_2 = get_oof(xg2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "et_oof_train, et_oof_test = get_oof(et)\n",
    "rf_oof_train, rf_oof_test = get_oof(rf)\n",
    "lr_oof_train,lr_oof_test = get_oof(Lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XG-CV: 0.8149660413971539\n",
      "RF-CV: 0.7683538163001293\n",
      "ET-CV: 0.7629770375161707\n",
      "lr-CV: 0.7661707632600259\n"
     ]
    }
   ],
   "source": [
    "print(\"XG-CV: {}\".format(accuracy_score(y_train,(xg_oof_train+0.5).astype(int))))\n",
    "print(\"RF-CV: {}\".format(accuracy_score(y_train,(rf_oof_train+0.5).astype(int))))\n",
    "print(\"ET-CV: {}\".format(accuracy_score(y_train,(et_oof_train+0.5).astype(int))))\n",
    "print(\"lr-CV: {}\".format(accuracy_score(y_train,(lr_oof_train+0.5).astype(int))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "l2_x_train = np.concatenate((xg_oof_train, et_oof_train, rf_oof_train), axis=1)\n",
    "l2_x_test = np.concatenate((xg_oof_test, et_oof_test, rf_oof_test), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dtrain = xgb.DMatrix(l2_x_train, label=y_train)\n",
    "dtest = xgb.DMatrix(l2_x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "xgb_params = {\n",
    "    'colsample_bytree': 0.5,\n",
    "    'silent': 1,\n",
    "    'subsample': 0.6,\n",
    "    'learning_rate': 0.01,\n",
    "    'objective': 'binary:logistic',\n",
    "    'max_depth': 3,\n",
    "    'num_parallel_tree': 1,\n",
    "    'min_child_weight': 5,\n",
    "    'eval_metric':['logloss','error'],\n",
    "}\n",
    "\n",
    "# res = xgb.cv(xgb_params, dtrain, num_boost_round=10000, nfold=5, seed=seed,stratified=False,\n",
    "#              early_stopping_rounds=20, verbose_eval=10, show_stdv=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tTrain-logloss:0.689565\tTrain-error:0.231646\n",
      "[50]\tTrain-logloss:0.556242\tTrain-error:0.199628\n",
      "[100]\tTrain-logloss:0.498686\tTrain-error:0.200356\n",
      "[150]\tTrain-logloss:0.46305\tTrain-error:0.195343\n",
      "[200]\tTrain-logloss:0.444521\tTrain-error:0.194009\n",
      "[250]\tTrain-logloss:0.433867\tTrain-error:0.193928\n",
      "[300]\tTrain-logloss:0.425812\tTrain-error:0.191462\n",
      "[350]\tTrain-logloss:0.420609\tTrain-error:0.190047\n",
      "[400]\tTrain-logloss:0.416362\tTrain-error:0.1894\n",
      "[450]\tTrain-logloss:0.414367\tTrain-error:0.188066\n",
      "[500]\tTrain-logloss:0.412262\tTrain-error:0.187662\n",
      "[550]\tTrain-logloss:0.410874\tTrain-error:0.187338\n",
      "[600]\tTrain-logloss:0.409709\tTrain-error:0.185074\n",
      "[650]\tTrain-logloss:0.408776\tTrain-error:0.184589\n",
      "[700]\tTrain-logloss:0.407881\tTrain-error:0.184145\n",
      "[750]\tTrain-logloss:0.407211\tTrain-error:0.183781\n",
      "[800]\tTrain-logloss:0.406546\tTrain-error:0.183659\n",
      "[850]\tTrain-logloss:0.406068\tTrain-error:0.183417\n"
     ]
    }
   ],
   "source": [
    "clf=xgb.train(xgb_params,dtrain,\n",
    "              num_boost_round=5,\n",
    "              evals=[(dtrain,'Train')],\n",
    "              verbose_eval=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ensembel model  the accuracy on the dev set is : 81.73%\n"
     ]
    }
   ],
   "source": [
    "test_pred = clf.predict(dtest)\n",
    "y_t = (test_pred+0.5).astype(int)\n",
    "acc =  accuracy_score(y_valid,y_t)\n",
    "\n",
    "print('ensembel model  the accuracy on the dev set is : {}%'.format(round(acc* 100,2)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
